{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T06:59:34.828117900Z",
     "start_time": "2024-07-29T06:59:25.101487900Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:12:55.712919Z",
     "iopub.status.busy": "2025-02-04T09:12:55.712431Z",
     "iopub.status.idle": "2025-02-04T09:12:58.895389Z",
     "shell.execute_reply": "2025-02-04T09:12:58.894621Z",
     "shell.execute_reply.started": "2025-02-04T09:12:55.712884Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=10, micro=14, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cu124\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-02T03:33:53.350580400Z",
     "start_time": "2024-05-02T03:33:52.447226500Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:12:59.511092Z",
     "iopub.status.busy": "2025-02-04T09:12:59.510416Z",
     "iopub.status.idle": "2025-02-04T09:13:02.744315Z",
     "shell.execute_reply": "2025-02-04T09:13:02.743182Z",
     "shell.execute_reply.started": "2025-02-04T09:12:59.511054Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2025-02-04 17:12:59--  https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
      "正在解析主机 storage.googleapis.com (storage.googleapis.com)... 142.251.215.251, 172.217.14.251, 142.250.217.91, ...\n",
      "正在连接 storage.googleapis.com (storage.googleapis.com)|142.251.215.251|:443... 已连接。\n",
      "已发出 HTTP 请求，正在等待回应... 200 OK\n",
      "长度： 1115394 (1.1M) [text/plain]\n",
      "正在保存至: ‘shakespeare.txt’\n",
      "\n",
      "shakespeare.txt     100%[===================>]   1.06M   439KB/s    用时 2.5s    \n",
      "\n",
      "2025-02-04 17:13:02 (439 KB/s) - 已保存 ‘shakespeare.txt’ [1115394/1115394])\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# !wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:00:39.596860100Z",
     "start_time": "2024-07-29T07:00:39.590864800Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:13:09.784351Z",
     "iopub.status.busy": "2025-02-04T09:13:09.783807Z",
     "iopub.status.idle": "2025-02-04T09:13:09.792781Z",
     "shell.execute_reply": "2025-02-04T09:13:09.792105Z",
     "shell.execute_reply.started": "2025-02-04T09:13:09.784313Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构造字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:01:12.053555500Z",
     "start_time": "2024-07-29T07:01:12.033478400Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:13:45.333183Z",
     "iopub.status.busy": "2025-02-04T09:13:45.332639Z",
     "iopub.status.idle": "2025-02-04T09:13:45.355942Z",
     "shell.execute_reply": "2025-02-04T09:13:45.355172Z",
     "shell.execute_reply.started": "2025-02-04T09:13:45.333147Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "# 去重，留下独立字符，并排序（排序是为了好看）\n",
    "vocab = sorted(set(text))\n",
    "print(len(vocab))  # 字典大小65\n",
    "print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:02:41.378489600Z",
     "start_time": "2024-07-29T07:02:41.356995400Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:14:27.887931Z",
     "iopub.status.busy": "2025-02-04T09:14:27.887421Z",
     "iopub.status.idle": "2025-02-04T09:14:27.892477Z",
     "shell.execute_reply": "2025-02-04T09:14:27.891735Z",
     "shell.execute_reply.started": "2025-02-04T09:14:27.887894Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 h\n",
      "1 o\n",
      "2 w\n"
     ]
    }
   ],
   "source": [
    "for idx,char in enumerate(['h','o','w']):\n",
    "    print(idx,char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:03:18.675580Z",
     "start_time": "2024-07-29T07:03:18.662577400Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:14:50.289918Z",
     "iopub.status.busy": "2025-02-04T09:14:50.289151Z",
     "iopub.status.idle": "2025-02-04T09:14:50.294799Z",
     "shell.execute_reply": "2025-02-04T09:14:50.293700Z",
     "shell.execute_reply.started": "2025-02-04T09:14:50.289881Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n"
     ]
    }
   ],
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:03:28.524626500Z",
     "start_time": "2024-07-29T07:03:28.515224100Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:15:08.175037Z",
     "iopub.status.busy": "2025-02-04T09:15:08.174487Z",
     "iopub.status.idle": "2025-02-04T09:15:08.180027Z",
     "shell.execute_reply": "2025-02-04T09:15:08.179165Z",
     "shell.execute_reply.started": "2025-02-04T09:15:08.174991Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n",
      " 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n",
      " 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n",
      " 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "source": [
    "# 把vocab从列表变为ndarray，比用idx2char更快\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:04:29.327737500Z",
     "start_time": "2024-07-29T07:04:29.183493900Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:16:42.835557Z",
     "iopub.status.busy": "2025-02-04T09:16:42.835051Z",
     "iopub.status.idle": "2025-02-04T09:16:42.968111Z",
     "shell.execute_reply": "2025-02-04T09:16:42.967360Z",
     "shell.execute_reply.started": "2025-02-04T09:16:42.835521Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1115394,)\n",
      "1115394\n",
      "[18 47 56 57 58  1 15 47 58 47]\n",
      "First Citi\n"
     ]
    }
   ],
   "source": [
    "# 把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:08:37.700926900Z",
     "start_time": "2024-07-29T07:08:37.659951700Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:17:24.967427Z",
     "iopub.status.busy": "2025-02-04T09:17:24.966933Z",
     "iopub.status.idle": "2025-02-04T09:17:24.973907Z",
     "shell.execute_reply": "2025-02-04T09:17:24.973011Z",
     "shell.execute_reply.started": "2025-02-04T09:17:24.967394Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11043"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1115394//101  # 把每101个截取做1个样本"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:13:46.372759500Z",
     "start_time": "2024-07-29T07:13:46.370233200Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:21:05.125025Z",
     "iopub.status.busy": "2025-02-04T09:21:05.124497Z",
     "iopub.status.idle": "2025-02-04T09:21:05.133472Z",
     "shell.execute_reply": "2025-02-04T09:21:05.132694Z",
     "shell.execute_reply.started": "2025-02-04T09:21:05.124990Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    #text_as_int是字符的id列表，seq_length是每个样本的长度\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1 #一个样本的长度\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len #样本的个数\n",
    "        \n",
    "    def __getitem__(self, index):#index是样本的索引，返回的是一个样本，比如第一个，就是0-100的字符,总计101个字符\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self): #返回样本的个数\n",
    "        return self.num_seq\n",
    "\n",
    "# batch是一个列表，列表中的每一个元素是一个样本，有101个字符，前100个是输入，后100个是输出\n",
    "def collat_fct(batch):\n",
    "    src_list = [] #输入\n",
    "    trg_list = [] #输出\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1])  # 输入\n",
    "        trg_list.append(part[1:])  # 输出\n",
    "\n",
    "    src_list = np.array(src_list)  # 把列表转换为ndarray\n",
    "    trg_list = np.array(trg_list)  # 把列表转换为ndarray\n",
    "\n",
    "    # 返回的是一个元组，元组中的每一个元素是一个torch.Tensor\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64)  \n",
    "\n",
    "# 每个样本的长度是101，也就是100个字符+1个结束符\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:17:14.428535500Z",
     "start_time": "2024-07-29T07:17:14.403947200Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:23:37.779139Z",
     "iopub.status.busy": "2025-02-04T09:23:37.778599Z",
     "iopub.status.idle": "2025-02-04T09:23:37.786916Z",
     "shell.execute_reply": "2025-02-04T09:23:37.786188Z",
     "shell.execute_reply.started": "2025-02-04T09:23:37.779101Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 100])\n",
      "torch.Size([64, 100])\n"
     ]
    }
   ],
   "source": [
    "for datas, labels in train_dl:\n",
    "    print(datas.shape)\n",
    "    print(labels.shape)  # 输出\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:25:07.262680600Z",
     "start_time": "2024-07-29T07:25:07.245109Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:27:45.774200Z",
     "iopub.status.busy": "2025-02-04T09:27:45.773679Z",
     "iopub.status.idle": "2025-02-04T09:27:45.794931Z",
     "shell.execute_reply": "2025-02-04T09:27:45.794184Z",
     "shell.execute_reply.started": "2025-02-04T09:27:45.774165Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================== 一层单向 RNN ===================================\n",
      "            embedding.weight            paramerters num: 16640\n",
      "            rnn.weight_ih_l0            paramerters num: 262144\n",
      "            rnn.weight_hh_l0            paramerters num: 1048576\n",
      "             rnn.bias_ih_l0             paramerters num: 1024\n",
      "             rnn.bias_hh_l0             paramerters num: 1024\n",
      "               fc.weight                paramerters num: 66560\n",
      "                fc.bias                 paramerters num: 65\n"
     ]
    }
   ],
   "source": [
    "class CharRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharRNN, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        #batch_first=True,输入的数据格式是(batch_size, seq_len, embedding_dim)\n",
    "        self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)  # 全连接只对最后一维做\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x) #(batch_size, seq_len) -> (batch_size, seq_len, embedding_dim) (64, 100, 256)\n",
    "        #这里和02的差异是没有只拿最后一个输出，而是把所有的输出都拿出来了\n",
    "        #(batch_size, seq_len, embedding_dim)->(batch_size, seq_len, hidden_dim)(64, 100, 1024)\n",
    "        output, hidden = self.rnn(x, hidden)\n",
    "        x = self.fc(output) #[bs, seq_len, hidden_dim]--->[bs, seq_len, vocab_size] (64, 100,65)\n",
    "        return x, hidden #x的shape是(batch_size, seq_len, vocab_size)\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "\n",
    "print(\"{:=^80}\".format(\" 一层单向 RNN \"))       \n",
    "for key, value in CharRNN(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:25:57.841628600Z",
     "start_time": "2024-07-29T07:25:57.832125500Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:29:34.015879Z",
     "iopub.status.busy": "2025-02-04T09:29:34.015353Z",
     "iopub.status.idle": "2025-02-04T09:29:34.020707Z",
     "shell.execute_reply": "2025-02-04T09:29:34.019870Z",
     "shell.execute_reply.started": "2025-02-04T09:29:34.015834Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16640\n",
      "262144\n",
      "1048576\n",
      "66560\n"
     ]
    }
   ],
   "source": [
    "print(65*256)\n",
    "print(256*1024)\n",
    "print(1024*1024)\n",
    "print(1024*65)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:27:35.393573400Z",
     "start_time": "2024-07-29T07:27:35.327680900Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:29:48.247263Z",
     "iopub.status.busy": "2025-02-04T09:29:48.246767Z",
     "iopub.status.idle": "2025-02-04T09:29:48.296167Z",
     "shell.execute_reply": "2025-02-04T09:29:48.295084Z",
     "shell.execute_reply.started": "2025-02-04T09:29:48.247226Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 100])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 100, 65])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_inputs = torch.randint(0, vocab_size, (3, 100))\n",
    "print(sample_inputs.shape)\n",
    "model = CharRNN(vocab_size)\n",
    "output=model(sample_inputs)\n",
    "output[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:57:28.992008300Z",
     "start_time": "2024-07-29T07:57:28.981317Z"
    },
    "ExecutionIndicator": {
     "show": false
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:29:52.349498Z",
     "iopub.status.busy": "2025-02-04T09:29:52.349001Z",
     "iopub.status.idle": "2025-02-04T09:29:52.355505Z",
     "shell.execute_reply": "2025-02-04T09:29:52.354584Z",
     "shell.execute_reply.started": "2025-02-04T09:29:52.349464Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 65])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[0].reshape(-1, vocab_size).shape  # -1表示自动计算该维度的大小，以保持总元素数不变。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T07:28:19.937860300Z",
     "start_time": "2024-07-29T07:28:19.932860Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:31:22.349904Z",
     "iopub.status.busy": "2025-02-04T09:31:22.349377Z",
     "iopub.status.idle": "2025-02-04T09:31:22.356774Z",
     "shell.execute_reply": "2025-02-04T09:31:22.356066Z",
     "shell.execute_reply.started": "2025-02-04T09:31:22.349859Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:03:04.330302Z",
     "start_time": "2024-07-29T08:03:01.989572500Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:32:02.857402Z",
     "iopub.status.busy": "2025-02-04T09:32:02.856612Z",
     "iopub.status.idle": "2025-02-04T09:32:04.268216Z",
     "shell.execute_reply": "2025-02-04T09:32:04.267371Z",
     "shell.execute_reply.started": "2025-02-04T09:32:02.857365Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算,如果数据集打乱了，stateful=False，hidden就要清空\n",
    "                # 如果数据集没有打乱，stateful=True，hidden就不需要清空\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失,交叉熵损失第一个参数要是二阶张量，第二个参数要是一阶张量，所以要reshape\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharRNN(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:32:05.881736Z",
     "iopub.status.busy": "2025-02-04T09:32:05.881029Z",
     "iopub.status.idle": "2025-02-04T09:35:21.817318Z",
     "shell.execute_reply": "2025-02-04T09:35:21.816454Z",
     "shell.execute_reply.started": "2025-02-04T09:32:05.881694Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17300/17300 [03:15<00:00, 88.30it/s, epoch=99]\n"
     ]
    }
   ],
   "source": [
    "record = training(\n",
    "    model,\n",
    "    train_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-02-04T09:36:30.110561Z",
     "iopub.status.busy": "2025-02-04T09:36:30.110045Z",
     "iopub.status.idle": "2025-02-04T09:36:30.275773Z",
     "shell.execute_reply": "2025-02-04T09:36:30.275017Z",
     "shell.execute_reply.started": "2025-02-04T09:36:30.110525Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWRpJREFUeJzt3Xd4VFX+BvD3TjKZFDKppJFCIJDQEgg1dKTDItgFVqxYFlcR2w/XXUVdYS2rrLKoq4INEVwBVxEIJSAQSjABQgkEQhIghfSeTGbO74+ZucmQgikwN8z7eR6eydy5d+bMl5uZN+eee64khBAgIiIiUhiVtRtARERE1BiGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIke2s34PcwGAy4fPkyXF1dIUmStZtDREREv4MQAqWlpQgICIBK1fJ+kQ4RUi5fvoygoCBrN4OIiIhaITMzE4GBgS3erkOEFFdXVwDGN6nVatvteXU6HbZt24ZJkyZBrVa32/N2NKwDawCwBmasA2sAsAZmba1DSUkJgoKC5O/xluoQIcV8iEer1bZ7SHF2doZWq7X5ndDW68AasAZmrANrALAGZu1Vh9YO1eDAWSIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSGFCIiIlIkhhQiIiJSJIYUIiIiUiSbDimr9qfjv2kqpGSXWrspREREdBWbDimbk7OxJ1uFzMJKazeFiIiIrmLTIUVlunS0QQgrt4SIiIiuZtMhRTLdGphRiIiIFMe2Q4oppQj2pBARESmOjYcUY0phRiEiIlIemw4pKnNPinWbQURERI2w6ZBSNyaFMYWIiEhpbDqkqHi4h4iISLFsOqSAA2eJiIgUy6ZDSt08KVZuCBERETVg4yHFeCs4dJaIiEhxbDqkSGBPChERkVLZdkiRx6RYtx1ERETUEEMKOHCWiIhIidoUUpYtWwZJkrBw4cJm11u/fj0iIiLg6OiIfv36YfPmzW152XYjn4Js5XYQERFRQ60OKYcPH8bHH3+MyMjIZtfbv38/Zs+ejYcffhiJiYmYNWsWZs2aheTk5Na+dLvhZG5ERETK1aqQUlZWhrlz5+I///kPPDw8ml13+fLlmDJlCp5//nn06tULr7/+OqKjo/Hhhx+2qsHtidfuISIiUi771my0YMECTJ8+HRMmTMAbb7zR7Lrx8fFYtGiRxbLJkydj48aNTW5TXV2N6upq+X5JSQkAQKfTQafTtabJjZJMB3pqa/Xt+rwdjfm9swasQf1bW8U6sAYAa2DW1jq0tX4tDilr167Fb7/9hsOHD/+u9bOzs+Hr62uxzNfXF9nZ2U1us3TpUixZsqTB8m3btsHZ2bllDW5Gbo4KgAonT53C5qKT7fa8HVVsbKy1m2B1rAFrYMY6sAYAa2DW2jpUVFS06XVbFFIyMzPx9NNPIzY2Fo6Ojm164eYsXrzYovelpKQEQUFBmDRpErRabbu9zi/FSUBBLsIjIjBtRGi7PW9Ho9PpEBsbi4kTJ0KtVlu7OVbBGrAGZqwDawCwBmZtrYP5SEhrtSikHDlyBLm5uYiOjpaX6fV67NmzBx9++CGqq6thZ2dnsY2fnx9ycnIsluXk5MDPz6/J19FoNNBoNA2Wq9Xqdt1Z7FTGITmSSmXTO6FZe9e3I2INWAMz1oE1AFgDs9bWoa21a9HA2fHjx+P48eNISkqS/w0aNAhz585FUlJSg4ACADExMdixY4fFstjYWMTExLSp4e2Bk7kREREpV4t6UlxdXdG3b1+LZS4uLvDy8pKXz5s3D126dMHSpUsBAE8//TTGjBmDd999F9OnT8fatWuRkJCATz75pJ3eQutxMjciIiLlavcZZzMyMpCVlSXfHz58ONasWYNPPvkEUVFR+P7777Fx48YGYccaOJkbERGRcrXqFOT64uLimr0PAHfddRfuuuuutr5UuzNfBZmTuRERESmPTV+7x3y8x2CwcjuIiIioAZsOKeaeFCIiIlIemw4pkunqPTzcQ0REpDw2HVJUPAWZiIhIsWw6pEgcOEtERKRYNh5SeAoyERGRUtl0SFFxMjciIiLFsumQUjdw1soNISIiogZsOqRw4CwREZFy2XRIMY+c5eEeIiIi5bHpkCL3pFi3GURERNQImw4p5glneQoyERGR8th0SJGvgsyMQkREpDg2HVI4mRsREZFy2XhIYU8KERGRUtl0SOHAWSIiIuWy6ZDCqyATEREpl02HFE7mRkREpFw2HVLAa/cQEREplk2HFBWvgkxERKRYNh1SOJkbERGRctl0SDH3pPAqyERERMpj0yFF4sBZIiIixbLxkMKrIBMRESmVTYcUTuZGRESkXDYdUjhwloiISLlsO6Tw2j1ERESKZeMhxXjLMSlERETKY9MhRcWeFCIiIsWy8ZBivOU8KURERMpj0yFFkngVZCIiIqWy8ZBivGVEISIiUh7bDimmWw6cJSIiUh6bDikcOEtERKRcNh1SJHngLFMKERGR0th4SDH1pFi5HURERNSQTYcUFa+CTEREpFg2HVIk8BRkIiIipbLpkMKeFCIiIuWy6ZDCgbNERETK1aKQsnLlSkRGRkKr1UKr1SImJga//PJLk+uvXr0akiRZ/HN0dGxzo9sLB84SEREpl31LVg4MDMSyZcvQo0cPCCHwxRdfYObMmUhMTESfPn0a3Uar1SIlJUW+bw4GSsDJ3IiIiJSrRSFlxowZFvf//ve/Y+XKlThw4ECTIUWSJPj5+bW+hdcRJ3MjIiJSrhaFlPr0ej3Wr1+P8vJyxMTENLleWVkZQkJCYDAYEB0djTfffLPJQGNWXV2N6upq+X5JSQkAQKfTQafTtbbJDRgMegCA3mBo1+ftaMzvnTVgDerf2irWgTUAWAOzttahrfWTRAuPdRw/fhwxMTGoqqpCp06dsGbNGkybNq3RdePj43H27FlERkaiuLgY77zzDvbs2YMTJ04gMDCwydd49dVXsWTJkgbL16xZA2dn55Y0t1kJVyR8lWqHnm4GLOhtaLfnJSIiIqCiogJz5sxBcXExtFpti7dvcUipqalBRkYGiouL8f333+PTTz/F7t270bt372tuq9Pp0KtXL8yePRuvv/56k+s11pMSFBSEvLy8Vr3JpmxMvIjnfziJoV3d8fXDQ9rteTsanU6H2NhYTJw4EWq12trNsQrWgDUwYx1YA4A1MGtrHUpKSuDt7d3qkNLiwz0ODg4ICwsDAAwcOBCHDx/G8uXL8fHHH19zW7VajQEDBiA1NbXZ9TQaDTQaTaPbt+fOYm9nZ/xBkmx6JzRr7/p2RKwBa2DGOrAGAGtg1to6tLV2bZ4nxWAwWPR6NEev1+P48ePw9/dv68u2Cw6cJSIiUq4W9aQsXrwYU6dORXBwMEpLS7FmzRrExcVh69atAIB58+ahS5cuWLp0KQDgtddew7BhwxAWFoaioiK8/fbbSE9PxyOPPNL+76QVOJkbERGRcrUopOTm5mLevHnIysqCm5sbIiMjsXXrVkycOBEAkJGRAZWqrnOmsLAQ8+fPR3Z2Njw8PDBw4EDs37//d41fuRGUNGcLERERWWpRSPnss8+afTwuLs7i/nvvvYf33nuvxY26UVRyT4p120FEREQN2fa1e3gVZCIiIsWy6ZDCqyATEREpl02HFMghhSmFiIhIaWw6pKh4FWQiIiLFsumQwlOQiYiIlMumQwoncyMiIlIumw4p5llSeAoyERGR8th2SJF7UphSiIiIlMamQwpPQSYiIlIumw4pHDhLRESkXDYdUngKMhERkXLZdEgx45gUIiIi5bHpkMJTkImIiJTLpkOKxKsgExERKZZNhxRzTwoHzhIRESmPTYcUc08KIwoREZHy2HZIMd1y4CwREZHy2HRI4cBZIiIi5bLpkMLJ3IiIiJTLpkMKJ3MjIiJSLpsOKWbsSCEiIlIemw4pKl4FmYiISLFsPKQYbzmZGxERkfLYdEjhwFkiIiLlsvGQIl17JSIiIrIK2w4pplv2pBARESmPTYcUTuZGRESkXDYdUngVZCIiIuWy6ZBSN5kbUwoREZHS2HRIka+CzIxCRESkOAwp4MBZIiIiJbLpkMKBs0RERMpl0yGFpyATEREpl22HFE7mRkREpFg2HlKMtzwFmYiISHlsOqTwKshERETKZeMhxXjLnhQiIiLlsemQUn9ECntTiIiIlMW2Q0q9gbPMKERERMpi4yGl7meehkxERKQsNh1SVPV7UqzYDiIiImqoRSFl5cqViIyMhFarhVarRUxMDH755Zdmt1m/fj0iIiLg6OiIfv36YfPmzW1qcHuqPyaFPSlERETK0qKQEhgYiGXLluHIkSNISEjALbfcgpkzZ+LEiRONrr9//37Mnj0bDz/8MBITEzFr1izMmjULycnJ7dL4tuKYFCIiIuVqUUiZMWMGpk2bhh49eqBnz574+9//jk6dOuHAgQONrr98+XJMmTIFzz//PHr16oXXX38d0dHR+PDDD9ul8W1Vf0wKQwoREZGy2Ld2Q71ej/Xr16O8vBwxMTGNrhMfH49FixZZLJs8eTI2btzY7HNXV1ejurpavl9SUgIA0Ol00Ol0rW1yA/ra2rrXrKmBvdTqcnRo5pq2Z207GtaANTBjHVgDgDUwa2sd2lq/Fn8rHz9+HDExMaiqqkKnTp2wYcMG9O7du9F1s7Oz4evra7HM19cX2dnZzb7G0qVLsWTJkgbLt23bBmdn55Y2uUk1esBcgq3btsHRrt2eukOKjY21dhOsjjVgDcxYB9YAYA3MWluHioqKNr1ui0NKeHg4kpKSUFxcjO+//x73338/du/e3WRQaY3Fixdb9MCUlJQgKCgIkyZNglarbbfXKausAg7tAQBMnDgRro7qdnvujkSn0yE2NhYTJ06EWs0asAa2WwOAdQBYA4A1MGtrHcxHQlqrxSHFwcEBYWFhAICBAwfi8OHDWL58OT7++OMG6/r5+SEnJ8diWU5ODvz8/Jp9DY1GA41G02C5Wq1u153FodYg/2xn377P3RG1d307ItaANTBjHVgDgDUwa20d2lq7Ns+TYjAYLMaP1BcTE4MdO3ZYLIuNjW1yDMuNZjlwliNniYiIlKRFPSmLFy/G1KlTERwcjNLSUqxZswZxcXHYunUrAGDevHno0qULli5dCgB4+umnMWbMGLz77ruYPn061q5di4SEBHzyySft/05aQcVTkImIiBSrRSElNzcX8+bNQ1ZWFtzc3BAZGYmtW7di4sSJAICMjAyoVHWdM8OHD8eaNWvw8ssv46WXXkKPHj2wceNG9O3bt33fRStxMjciIiLlalFI+eyzz5p9PC4ursGyu+66C3fddVeLGnWjWF67x3rtICIiooZs+to9kiRBMl21R/DqPURERIpi0yGlPh7tISIiUhabDynmQz4MKURERMrCkGK65cBZIiIiZWFIMd0yohARESkLQ4rp1sDTe4iIiBSFIUW69jpERER04zGkmG45JoWIiEhZbD6kmFMKj/YQEREpi82HFHMBeIFBIiIiZbH5kGLGnhQiIiJlsfmQUjdwlimFiIhISRhSTLfsSSEiIlIWhhTTLYekEBERKQtDinx2D1MKERGRkjCkmG4ZUoiIiJSFIcV0y4xCRESkLDYfUswphSGFiIhIWWw+pNRdBZkphYiISEkYUky3PAWZiIhIWRhS5MM9TClERERKwpBiumVPChERkbIwpJhu2ZNCRESkLAwp5sM91m0GERERXYUhxXRr4PEeIiIiRWFIYU8KERGRItl8SDHjtPhERETKYvMhxXy4h10pREREysKQYrrlkBQiIiJlYUgxpRQe7iEiIlIWhhTTLSMKERGRsjCksCeFiIhIkRhSzD8woxARESmKzYcUM/akEBERKYvNhxSVfBVk67aDiIiILNl8SDFjTwoREZGy2HxI4TwpREREysSQwpGzREREisSQYrplTwoREZGyMKSYbjkkhYiISFlaFFKWLl2KwYMHw9XVFT4+Ppg1axZSUlKa3Wb16tWQJMnin6OjY5sa3Z44mRsREZEytSik7N69GwsWLMCBAwcQGxsLnU6HSZMmoby8vNnttFotsrKy5H/p6eltanR7kkxjURhRiIiIlMW+JStv2bLF4v7q1avh4+ODI0eOYPTo0U1uJ0kS/Pz8WtfCG0SwJ4WIiEhR2jQmpbi4GADg6enZ7HplZWUICQlBUFAQZs6ciRMnTrTlZduVxMnciIiIFKlFPSn1GQwGLFy4ECNGjEDfvn2bXC88PByff/45IiMjUVxcjHfeeQfDhw/HiRMnEBgY2Og21dXVqK6ulu+XlJQAAHQ6HXQ6XWub3IBOp5NTmq62tl2fuyMxv29bff8AawCwBmasA2sAsAZmba1DW+sniVYe53jiiSfwyy+/YO/evU2GjcbodDr06tULs2fPxuuvv97oOq+++iqWLFnSYPmaNWvg7OzcmuY2aeVJFU4XqzC3ux5DfNidQkRE1F4qKiowZ84cFBcXQ6vVtnj7VoWUJ598Eps2bcKePXsQGhra4he96667YG9vj2+//bbRxxvrSQkKCkJeXl6r3mRTdDod7vjXDpwqUmHZbX1wR3SXdnvujkSn0yE2NhYTJ06EWq22dnOsgjVgDcxYB9YAYA3M2lqHkpISeHt7tzqktOhwjxACf/7zn7FhwwbExcW1KqDo9XocP34c06ZNa3IdjUYDjUbTYLlarW73ncU8T4pKZWfTOyJwferb0bAGrIEZ68AaAKyBWWvr0NbatSikLFiwAGvWrMGmTZvg6uqK7OxsAICbmxucnJwAAPPmzUOXLl2wdOlSAMBrr72GYcOGISwsDEVFRXj77beRnp6ORx55pE0Nby/ywFmehExERKQoLQopK1euBACMHTvWYvmqVavwwAMPAAAyMjKgUtWdNFRYWIj58+cjOzsbHh4eGDhwIPbv34/evXu3reXthNPiExERKVOLD/dcS1xcnMX99957D++9916LGnUjcVp8IiIiZeK1ezgtPhERkSLZfEgx44yzREREymLzIcVcAEYUIiIiZbH5kGIelGLgyFkiIiJFsfmQIg+ctWoriIiI6GoMKaZbdqQQEREpC0OKfBVkphQiIiIlsfmQYsaMQkREpCw2H1LMBeA8KURERMpi8yGl7to9REREpCQ2H1LM2JNCRESkLDYfUnjtHiIiImViSOHZPURERIrEkGK6ZUYhIiJSFoYU0y0ncyMiIlIWhhTztXvYlUJERKQoDCmmW0YUIiIiZWFIMd1y4CwREZGyMKTIZ/dYtx1ERERkyeZDihnHpBARESmLzYcUTotPRESkTAwpplv2pBARESkLQ4rplhmFiIhIWRhSOC0+ERGRIjGkmG454ywREZGyMKSYbtmRQkREpCw2H1LAafGJiIgUyeZDis0XgIiISKH4HW3CnhQiIiJlsfmQwqsgExERKRNDimmuWWYUIiIiZWFIMd3yFGQiIiJlYUgxpxRevYeIiEhRGFJMtwaDVZtBREREV2FI4cBZIiIiRbL5kKI2VaBCp7duQ4iIiMiCzYcUJzvjbVlVrXUbQkRERBZsPqQ4mkJKaZXOug0hIiIiCwwp9sbbUvakEBERKYrNhxQnO+OA2bJqhhQiIiIlsfmQUne4hyGFiIhISVoUUpYuXYrBgwfD1dUVPj4+mDVrFlJSUq653fr16xEREQFHR0f069cPmzdvbnWD25s5pJRV18LAaWeJiIgUo0UhZffu3ViwYAEOHDiA2NhY6HQ6TJo0CeXl5U1us3//fsyePRsPP/wwEhMTMWvWLMyaNQvJycltbnx7cLKv+7mshr0pRERESmF/7VXqbNmyxeL+6tWr4ePjgyNHjmD06NGNbrN8+XJMmTIFzz//PADg9ddfR2xsLD788EN89NFHrWx2+7GXALWdBJ1eoKyqFlpHtbWbRERERGhhSLlacXExAMDT07PJdeLj47Fo0SKLZZMnT8bGjRub3Ka6uhrV1dXy/ZKSEgCATqeDTtd+pwrrdDpIEtBJY4/CCh0KyyrR2aVNJemQzDVtz9p2NKwBa2DGOrAGAGtg1tY6tLV+rf5GNhgMWLhwIUaMGIG+ffs2uV52djZ8fX0tlvn6+iI7O7vJbZYuXYolS5Y0WL5t2zY4Ozu3tslNUulrAEiIjfsVqa7t/vQdRmxsrLWbYHWsAWtgxjqwBgBrYNbaOlRUVLTpdVsdUhYsWIDk5GTs3bu3TQ1ozOLFiy16X0pKShAUFIRJkyZBq9W22+vodDrExsbCx8MV+dll6DtgMMb07Nxuz99RmOswceJEqNW2ebiLNWANzFgH1gBgDczaWgfzkZDWalVIefLJJ/HTTz9hz549CAwMbHZdPz8/5OTkWCzLycmBn59fk9toNBpoNJoGy9Vq9XXZWVxN41AqamHTO+P1qm9HwhqwBmasA2sAsAZmra1DW2vXorN7hBB48sknsWHDBuzcuROhoaHX3CYmJgY7duywWBYbG4uYmJiWtfQ6cjVNO8vr9xARESlHi3pSFixYgDVr1mDTpk1wdXWVx5W4ubnByckJADBv3jx06dIFS5cuBQA8/fTTGDNmDN59911Mnz4da9euRUJCAj755JN2fiut56oxloHX7yEiIlKOFvWkrFy5EsXFxRg7diz8/f3lf9999528TkZGBrKysuT7w4cPx5o1a/DJJ58gKioK33//PTZu3NjsYNsbrZO5J4VT4xMRESlGi3pShLj2jKxxcXENlt1111246667WvJSN1RdTwpDChERkVLY/LV7AMCFIYWIiEhxGFJQN3CWY1KIiIiUgyEF9c7u4ZgUIiIixWBIgXFafICHe4iIiJSEIQU83ENERKREDClgTwoREZESMaQA8HA2TttbVKmDwXDt06yJiIjo+mNIAeDh7AAA0BsEiip5yIeIiEgJGFIAONir4OZk7E3JL6u2cmuIiIgIYEiReXcy9qbkldVYuSVEREQEMKTIvDppAAD55exJISIiUgKGFBNzT0o+e1KIiIgUgSHFxMvF1JPCMSlERESKwJBi4mUek1LOnhQiIiIlYEgxkceksCeFiIhIERhSTLxdOCaFiIhISRhSTOrO7mFIISIiUgKGFBN5TEopD/cQEREpAUOKibepJ6W0uhZVOr2VW0NEREQMKSZaR3uo7SQAQAEP+RAREVkdQ4qJJEnyXCl5PMOHiIjI6hhS6vFzcwQAXC6qtHJLiIiIiCGlnhAvZwBAen6FlVtCREREDCn1BHsaQ0pGAUMKERGRtTGk1MOQQkREpBwMKfUwpBARESkHQ0o9IV4uAIBLhZWo1Rus3BoiIiLbxpBSj4+rBg72KtQaBC4XVVm7OURERDaNIaUelUriIR8iIiKFYEi5ijmkpBeUW7klREREto0h5SrmuVLO5pRZuSVERES2jSHlKgNDPAAA+8/lWbklREREto0h5Sojw7whScCZnDJkFXN6fCIiImthSLmKu7MDIgPdAQBxKVdgMAjrNoiIiMhGMaQ0YkwPbwDA4h+OY8Q/dqK0SmflFhEREdkehpRGTOztB0ky/pxVXIWE9ELrNoiIiMgGMaQ0ol+gGzb8aQS6dTbOQJt8sdjKLSIiIrI9DClN6B/kjjlDggEAe1PzsGDNb/j5WJaVW0VERGQ77K3dACXr28UNAHAwrQAAcCitAFP6+sFOJVmzWURERDaBPSnN6BOgtbh/pbQah0yBhYiIiK4vhpRmuDqqGyz76dhlK7SEiIjI9rQ4pOzZswczZsxAQEAAJEnCxo0bm10/Li4OkiQ1+Jednd3aNt9Qk3r7AgAC3BwBAL8kZ6NWb7Bmk4iIiGxCi0NKeXk5oqKisGLFihZtl5KSgqysLPmfj49PS1/aKt6Y1Revz+yD2EVj4OGsRkF5DX7LKLJ2s4iIiG56LR44O3XqVEydOrXFL+Tj4wN3d/cWb2dtPlpH3BfTFQAwpmdnbEy6jJ2nczEk1NO6DSMiIrrJ3bCze/r374/q6mr07dsXr776KkaMGNHkutXV1aiurpbvl5SUAAB0Oh10uvab/dX8XL/3OUeFeRlDyqkcPDuhe7u1w9paWoebEWvAGpixDqwBwBqYtbUOba2fJIRo9cVpJEnChg0bMGvWrCbXSUlJQVxcHAYNGoTq6mp8+umn+Oqrr3Dw4EFER0c3us2rr76KJUuWNFi+Zs0aODs7t7a5bVauA/6SYAcBCa9E1+JciQRHO6CfJ6/vQ0REdLWKigrMmTMHxcXF0Gq1197gKtc9pDRmzJgxCA4OxldffdXo4431pAQFBSEvL69Vb7IpOp0OsbGxmDhxItTqhmfyNObe/xzCkYwiTO7tg60nc2GnkrDv+dHw6qRpt3bdaK2pw82GNWANzFgH1gBgDczaWoeSkhJ4e3u3OqRYZTK3IUOGYO/evU0+rtFooNE0/NJXq9XXZWdpyfPOHhqCIxlF2HoyFwCgNwhsO52HeaZxKx3Z9apvR8IasAZmrANrALAGZq2tQ1trZ5V5UpKSkuDv72+Nl26zW6MC4Kd1tFj2YxLnTiEiImpvLe5JKSsrQ2pqqnw/LS0NSUlJ8PT0RHBwMBYvXoxLly7hyy+/BAC8//77CA0NRZ8+fVBVVYVPP/0UO3fuxLZt29rvXdxADvYqPDIqFG/8fAr9urgh+XIxEtILcbGwAoEe1hsvQ0REdLNpcUhJSEjAuHHj5PuLFi0CANx///1YvXo1srKykJGRIT9eU1ODZ599FpcuXYKzszMiIyOxfft2i+foaB4aEYpADycMCfXC418fwaG0Auw8nYt5MV1RpdOjWmeAmzO7B4mIiNqixSFl7NixaG6s7erVqy3uv/DCC3jhhRda3DAlU6kkTOlrPFw1NrwzDqUVYM+ZK5gX0xXzv0zAb+mF2PrMaPasEBERtQGv3dNGo3t0BgDEn8vH2ZxS/Ho2D+U1euw8nWvllhEREXVsDClt1NtfCy8XB5TX6PHGz6fk5QfO51uxVURERB0fQ0obqVQSRvbwBgDsPnNFXn7gfAEMBk7yRkRE1FoMKe1g7tAQ+Wc7lQRHtQoF5TU4k1tqxVYRERF1bAwp7WBIqCdinxmN2wZ0wV+n98LgrsaLD87+5ADe+OkkdHoDrpRWY/YnB7AuIdPKrSUiIuoYrDLj7M2oh68r3runPwDAzVmNval5KKzQ4dO9aTiTW4YhXT0Qfz4fl4srMbN/AC4XVSHU28W6jSYiIlIwhpTr4LYBgbgl3Bd7zl7BC98fw54zV5CYXggASM+vwGNfHUFcyhU8OKIrXp7eG3YqycotJiIiUh4e7rlO3JzVmBEVgLlDgwEApdW18mNxKcYBtqv2XcC/dpy1SvuIiIiUjiHlOrtrUFCzj289kS3/XKXT45VNyVh3mONWiIiIGFKus3A/V0QGugEAfLV1V3ae2tcPAHA2twwVNcZeltd/Ookv4tPxwn+PobpWj+ziKgxfugNL/nfixjeciIjIyhhSboAlt/bB9Eh/eWAtAMyL6QpfrQZ6g0DsyRz8dWMyvjlYd82jxIwiLN9xFpeLq7Bq34Ub32giIiIr48DZG2BAsAdWzPGAEAJT+vihqlaPwV09EBnojtiTOXh6bVKDbfafy8eZnLp5VgrKa+Dp4nADW01ERGRd7Em5gSRJwkf3DcTqB4fA3k6F/kHu8mOeLg5Y9eBgLLu9HwBgx6kcHL9YLD+emlt2o5tLRERkVQwpVhQV6C7//PiYbhgX7oMRYcYp9k9cLkGN3iA/fu5KGYordHh6bSK21RtsS0REdLNiSLGi/sHu8HJxQLCnM+4b1hUAEOTp3Ogkb6m5ZVi1Pw2bki7j2fVHUVBec4NbS0REdGNxTIoVddLYY+ezYyGpACcHO3n5Z/cPwo9HLyOnpArenTT4YGcqzuaW4fwV4yGf0qpaLPvlFO4ZHIwD5/MR6OGEsT194OasBgD8evYK/N0cEebjapX3RURE1B4YUqzMHCzq69a5ExZO6AkAOJRWgA92pmKP6QrLajsJOr3AuoSLWJdwUd4m2NMZu54bi5TsUtz32SGoJCDh5YkcbEtERB0WD/coXPfOlod+7hwYhL9M64Vu3i7opLHHxN6+cHawQ0ZBBZIyC7HzdA4AwCCAd7elAACOZhYhv6z6hrediIioLdiTonBenTSI8HPF6exSRAW546nxYfB3c8L80d3kdZ76NhE/Hr2MbSdzkJheJC9fcygDpVW1+PHoZUQFumHjghGQJF4niIiIOgaGlA7gu0djUFKlQ5Cnc6OPT+ztix+PXsaG3y7JA2pH9+yMPWeu4MejlwEARy8W49jFYkQFuaOyRg9HtYqBhYiIFI2HezoAN2d1kwEFAMaGd4baTkJuaTVqDQKh3i5Y9cBg3Ds4CPVzyMwV+zDunTj0+tsWvLQh+Qa0nIiIqPUYUm4Cro5qTO/nL9+f1NsXdioJy+6IxNFXJuHb+cPkx9LyygEA3x7KwJH0Anl5RS1QXu9KzURERNbGkHKT+Ofd/bH5qVFY9eBgPDOxp7xc66jGsG6eGN7dC51dNfj7bX1x+4AuAIBXfjyBnJIqPPf9cfzlsB3u/PggautNIEdERGRNHJNyk1CpJPQO0KI3tA0ekyQJ3zwyVB6DMqm3H2JP5SD5UgnGvh2HSp0egITUK+XYdjIHDnYqjAjztpi75WpCCHy4MxXVtQYsGBcGJwc7VNfqUVBeA1dHNTppuGsREVHb8JvERtQfJNvZVYN/zR6Ah1cfRqVOD19XDTqhCudKJfzpm98AAHdEB+KJsd2QWViJsT07Q5IkZORXoEJXiwg/LTYmXcK7sWcAAFtPZOPT+wfh7o/jkVNSDSe1HbY9M7rZcTRERNS+tp7IRphPJ3Tv3MnaTWk3DCk2aly4D5bfOwC/JGfhmfHdsStuN95Mqtsdfki8iM3Hs1Cp0+Pl6b0Q090Ld38UD51B4IcnhuP1n04BABzsVDibW4aHv0hATolxLpZKnR77UvNw75Bgq7w3IiIlE0LgcnEVvFwc4Khu2GNdXKnD4h+OYVJvP8wyHZ5vyi/Hs3D8UjFuifDBY18dAQCceWMqHOxVOHm5BOeulOG/v13E2ZwyfPnwEHh30kAlGccydgQMKTZsRlQAZkQFQKfTwdcJGNPTG/tS89GtswvO5JSZDgMBf998Ck5qO1TUGO8/sOowCspr0NO3Ex4Z1Q0vfH+swVWa95/Lx6d70xAd7I637oyyeExvEMgtrYK/m9ONeaNERAqRlFmER75IQF5ZNaIC3bDpyZFIvlSMjYmXMLKHN8aG++Cbg+nYfDwbm49no6evK3oHaFFdq8fc/xyEs8Yeqx4YDDuVhPhz+XjC1Pt9MK3uRIgfj16Gn9YRf/zsoMVrf7H/ArYkZ6O4Uoc/DgvB42O6w9PFASoJip2SggNnSbZidn/sX3wL/j13IJwd7NDTtxNmDwmGEJADCgDkmWavfWFyBGZEBliMP1k4oQcA4H/HLiM1twzrEi7iy/gLeGDVITnIvPa/E4hZuhP7UvNu4LsjIrqxvoq/gBHLdiLhQl2AWJeQKX+GHr1YjE9/PY8/fLAXn+5Nw//99zgAYOuJHHn9Z9cfRa3egB2ncpGQXmia/+oSiipq8NTaRHm9I+mF8s//2XMen/x6HgDQzdsFjmrjV/23hzKQW1qN6loDPtubhpilOxD2l80Y+Y9deHvraVwsrLh+xWgl9qSQTGOvQicnNXxcgX0v3gInBzs4qu3w2OhuKKioQU9fVwxfugMlVbXo5a/F+F4+kCQJf4j0x9rDmejbRYvp/fzx/vazEKLuef+26QQAQG13Gv+4IxJfxKcDAL47nIkRYd7WeKtERNeVEAJ/NX323flRPI69OglaRzUOnM+3WO+tLSnyz9klVUjMKMTRzCJ52amsEmxKuoyfjl2Wly3ffhaHLxTiSmnjlztJySlFSk4pJAlY9eBgqCQJo97aBZ3e+ME8PsIH+eU1SDK9zqWiSqzYdQ5DQr0Q6KGssYTsSaFGedQ7VtrV2wXRwR7opLHHQyNDYaeS8MKUcLl7cMG4MIwL74yXpvZCqLcLNPaN71a7Tufii/0X5Pt2qrruxeziKmxIvIjiCt31e1NERG2QVVyJP31zBL9lFF5z3fOmOanM3vz5FHJLqnD+SjkkyXhyAgDUmKZ9cLAzfm6+Y7rm2rBunnhxSoS8bLfpIrOuGntcyK/AmoMZAIC/39ZXfg1JApbd3k++P7ZnZ4R4uSDI0xmBHnWH1x8d3Q0b/jQccc+NRfziW/DB7AGY1T8AI7p7tawgNwBDCrXI0+N74MSSyRgX7iMvC/J0xqoHh2B4mDfs7VSI8HMFADg72GFwVw+4ORkHaNUaBJbvOCtvl1lg7Fr8z57zGL5sB5757iieXZ/U5GtfyCvHoDe2442fTl6Hd0ZEZKlcByxafwx7zxoPTX+8+zw2H8/GP3453eQ2QggkXyrGj0nGng8H0x9t6xIy8fUBYy9yb38tJvSq+wwN8+mEqf38AAD7Uo09LdP7+WNeTAi8XByQVVwFgwAGhnhgxdxo+LhqABgn7pwzJBjenYz3u3m74N4hwXhjVl+Eertg4YS6ObOGdTMGEK2jPQaGeECSJHT1doG/mxNmRAXg/XsHwN5OeZGAh3uoRSRJanQ0en29A7Q4erEYw7p54ZP7BkKnF1gZl4p/7Uy1WC+zsAK5pVV4e1sKDKbDQ9tP5eJ0dgki/BrO9/LVgXTklVXjywPp+PP4HnL4ISJqKYNBIPlyMfoEuMHcqXv14NE92SpsuZiNzMIqjAjzws7TuQCAhPRCFJbXwMPFwWL9zIIKvPjfY9h/ru6QzguTw3EorQDbTubIn4HDunlhUFdPeZ1bInzg3ckBm0zBRpKAyX394KKxx5u398MX+y+gq7cLHhrRFWE+rtj13FjsP5ePkWHekCQJQ0I9sPl4NqIC3QEAfxwWgj8OC7Fo29S+fvj+yEXMiApQZBhpCkMKtbu5Q0Nw4nIJHh/THfZ2KtjbAXOGhmD7qVz09O2Ep8b3wC3v7kZOSTX+vescamoNiA52h5+bIzYfz8ZHcefw/r0DUFBeAw9nNSRJgk5vwMbESwCAmloDvjucAVdHNbKLq3Br/4Cbal4AIvr94lJykZZXjvtjukKl+v1nqHy+Lw1v/HwKz03qiaTMIuw/l4/7YkKwaGJPbEnOxpWSShwrMD7fyawSpOSUIsPU+6s3CDy3/ijC/Vzx1PgeOJlVAi8XBzy5JhHHLxVbvM64CB+MDe+MHadzoTf9NTYu3AedXTXo5a/FqawSTO7jh2pd3ckJQ0M94ePqCACY3McPk/v4WTyni8YeE3v7yvcfHhmK9PwK3BdjGUzqG9/LF7HPjEawl7LGnFwLQwq1u75d3PDjkyMtlvm5OWLz06MAGLtDnR2MpzSvNo1RefKWMHTuZAwpPx69jE6O9vj6QAbemNUXfxwWgriUK8g3XeEZAN7cXNfdevhCAdbUuz4RESlHUUUN3tx8Cn8cFoJI01/611JYXoMX/3sMdwwMbPAFXV92cRUe++oIqmsN6OyqwR8iAxqsk5hRCAEgOtjDYvn6hIsAgC/i0+UBqB/vPo+80hpsSLxo6t01hpSaWgM+ijtnsf2O07nYcToX8efzkZhRBEkChDAeTvnhTyNwKK0ATg4q+Q+odY8Nw5mcMgR6OGFEmPHQyyf3DURmYQUGhnhYjMerfy2232NgiCd+fmrUNdfr4evaoudVgo7T50M3DUmSEFRvBHmwpzPGhfugX6AbJvTygUEAXx8wDgr7fG8aAMgj2yf38W3wfAkXClFRU4vjF4sx5f092JKcdQPeBRH9Hl8fSMe6hIt4wzQB5O/xXUImtp3MwUs/HEdVvR6Gqy3fcRbVtcaBp//acRYGU0/F+9vPIPzlX7B6Xxru/Cget/97P97aclo+9Tc1twwpOaUA0OAMmf/+dlE+/FzfRtOhmNuvmlwtMaMIAOQzGl+e3hthPp0wZ2gwbhsQKK83MMQTs4cEY1SPzvJhpSBPZwzvbjzD0c1ZjRFhXvBx1WBaC0PKzYw9KWQVQZ7O8ofE5D6+8i/tc5PDseN0rvwLX1Spg8Eg5IFrD44IRWSgO7KLq/DClHBMef9XXCqqxKG0AnywMxWns0uxYtc5TOlr/CWvqKnF0s2nMairB2b27wKDQeCbg+kYHOrZ6LgXIqpTVl2Lj7an4o7oQPSs91e4Tm+AvUr6XROAmQ9/HMkoRHGlrtmxZPllxjk8zHMo5ZfX4L+/XcTdg4Lww28XMb6XrzxI9FJRJdYlZAIwTp9wJqcMm45eQrivFh/sTIXeIPDq/+oG2f877hz+HXcOvf21cHVs+NX30rQIfL73ArJLqgAAjmoVqnQG+Ls5IqvYuMzBToX/mxqB6BAPeHdyQOzJXPz3t4sYH+GDOwcGorhSh7sGBTZ47t/rq4eGQmcwQGPf/Lg/W8KQQlbh56aRf67fnRvhp8Wfx4VhZ0ouki+VoKC8BttP5SC/vAbODnaIDvaQR6kDwIgwL6xLuIh/bEnBqawSAMYPxctFlQhwd8Ki745iy4lsfHUgHdP7+WND4iX8ddMJeabHyho9Ptp9DhMirj1fS2WNHlU6fYPBckQdXUp2KYordRgS6mmxfNW+dHy8+zzO5pTh8wcGAwBOXC7G/Z8fwsAQD3x83yDo9AZ8sDMVQ0M9G533KPmS8fdSbxDYl5rXZC9Bda0et364D7mlVfJ8HgDw6a9p0NUa8Or/TmJGVD7mxYTIE5fpDQJDunpidE9vvLPtDP628QR83RzlsR+AcRDqMxOM40xOZpXgpOlzAgB6+nbCmRzjJJNT+/qjSmfAP2PPwNXRHj8/ORyfb9qFgQPD8ee1RwEAtw3oAh+tozwo9ZYIX0zr54cRYd7XPKHg91CpJGhUDCj1MaSQVajq/QV29bHiRZPCsWhSOGZ8sBfHLxVjhelY8NBQT/l0PrMRYd5Yl3BRDihm20/lwM1JjS0nsuVlSZlF2GAafHvicgmqdHqs3H0O/9pxFt8cTMeCnkBJpQ5alR3e2nIaYT6dcM9g4/WHhBC4++N4XMgrx8YnRzQ5UFcIodjppcm21eoN2H4qByPCvC2u26I3CNz10X6UVNXipz+PRN8ubvJjsaeMZ7Mkm3pDsoorcduK/ajRG7D1RA6qdHr87+hl/GvHWXh3csCBxeNRVWvAqz+ewMz+AegT4IZLRZXy88Wl5MLLxQHPfX8U0/r6Y9GknnKvwZbkbIt1vVwcUF5Ti7S8cnxtmhNkz5krSLhQgKziKnlekWn9/ORxawnphSjNLYO7sxovTI7AXzcl487oQDw1vgeeGt8DV0qrsS81D3vOXoHBIPDE2DDMWrEPUUFuCPJ0xv3DuyIluxQTevvA380RA7wFBgTV1WP+6FCLmjrYqzC+V8ND0NR+GFLIKuaP6oa4lCt4cETTI/L7Bbrh+KViefbFkT06N1hnRJg3HOxVqKk1YECwO4aGeuGj3eewKelygymevzuciXjTbI+1BoHfMgrxjWnegryyGiz5zR5vHo3DxN6+2HzcGG5G9eiML/ZfwLgIH7nb+qUfjmPto8MswkhNrQF3fbQf9nYqfP7AYJ4eTVZzuagSBiEazBz6982nsGrfBdwfE4IlM+smADuVVYKSqloAwOr9F/DGrL6AQaCgGjiVbTwkm1tajSul1Xh7a4o8+RgAnM0pk0+bzSurwb5z+biQV47vj1zE2ZxSPDspHACgkgCDAHaevoLzV8qRWVCJj/ecx9ncuh6ab0zj0MzC/VwhScZ5Q8yX1Ciu1KG40jjA1NyOSX38YG+nwgdzBuCF748h2NMZT94SBn83J0zr52cRyDq7ajBrQBeLi/b9+uI4ODsYg5Kbkxor5kYDAHQ64+v4ah3x/j39YaeSEObT8QaednQMKWQVQZ7O2PPCuGbXiQp0wxrT9bHsVBLGhTcMKd6dNPh2/jBU1NRiZJg3Mgoq8Mmec3J3cJCnEx4b3R0vb0zG+iMXLbZ946dTyC+vQWdXDap0epRW1UKnF3JAAYD7PjuIc1fK8fGe8/Kyg2kF2Hk6F0UVOlTV6jF3aAiSMotw9KIxxDz6ZQK+engoHOxVWJeQid/SC7FkZh/5L8bNx7Pw7rYUfDgnGr38G46L0RsEKnV6i2siUcd1JqcUL3x/DE+P74FxET7X3qAVhBCo0hmgMxgw/V+/oqJGj7WPDsMAUy+l3iCwat8FAMazWeqHlPrXlfn+yEV8f+Qi/jg0CJUFln88bD+VI09Q5qqxR2l1LXafycX+c3XX4NqUeAl604Cys7llcrCf2NsXiRlFyC2tlgevAsDO07nIyK9AWn45Dl0ogJ1KwpienbHzdC7mxXTF2ZxSeXKzxkQFuiHA3TiTqr+bE756eKjF4+7O1z40ax7j0pxrXYmYrp8Wn92zZ88ezJgxAwEBAZAkCRs3brzmNnFxcYiOjoZGo0FYWBhWr17diqaSrYnp5g17lQRPFwesmDMA3Zo4xDIwxEMeMR/i5YL37ukvT82/eGovi/kEAONsjwDkY9MPjwxF7MKRWBxViyAPyyszn7tiObW12dcH0vHc90fxlw3JyMivwMF61+MwhpgclFXX4m+bkrH2cCZ2mLrNa/UG/Omb33DuSjne3Xam0ed+bv1RDHoj9ndNvV1feXUtVuxKxfkrZc2ul5JdisJ6p3N3RFU6Pfal5lmMPVCalOxSnL9ShlX70pCUWYT/++FYs2eqtNb7288g+vVY9HllC55ck4jCCh2qaw2Y/+URxJ7MMQ48v+pinhU1tfLPhy803M++PpiJA7nG3yF7U0/n4h+Oo9YgMCLMC/cMDgIAvLPtDAwC8O5kDANbTmTLg14ravTYajrcGh3sgSfGdpeff1QPb4w0jV95eVMyHv0yAQBwR3QXfHLfQGx7ZjSm9PXDsCamab81KgAaexUeHBHa6ON082hxSCkvL0dUVBRWrFjxu9ZPS0vD9OnTMW7cOCQlJWHhwoV45JFHsHXr1hY3lmxLsJczdj03FrufHyufrfN7zOzfBVsXjsaaR4ZiWj9/+GodMam3L3xcNfj33Gg8PzlcXrezqwb3DTNOPe3nDLx/dySiAt3gq238ryvztrtSrshnIB2+UIADaZZ/7SVmFGFrcjaqdMYu6V9NZydtO1l3ddPcUuMZAxsSL+LBVYeQW1KFrOJKbEq6hCqdAa9sOiGfUvl7PPNdEt7emoKXNhxvcp3DFwow+f09eMT0pdBRLd9xFnM/PYhvD2U0u561QkxheQ1u+/c+zFyxD9tNATWnpBpfmS6uabZoXRJmrdjXbHjZezYPK+POobbeYRaz2JM5eH/7WRRW6GAQxjEbAODiYIe8smrM/zIBy3ecxfdX9SLe/u/9GPRGLG779z78fNx4yv6rM3pj7tBgeZ1LFRLUdlKDCcKeGBPWoAfw6Qk90dXLGRU1euSV1QXgY6bexf5B7pg9JFiezv3uQUH4Q6Txd3rPmSuorjVgQi8fvDGrH+ztVPKZRJGBbvIVfEf1MIYajb0Kf7+tL06/PoU9HDagxSFl6tSpeOONN3Dbbbf9rvU/+ugjhIaG4t1330WvXr3w5JNP4s4778R7773X4saS7QnydLY4pvx7dfV2wfB6Zxp8fN9AHFg8HtP6+SMysG4g3MIJPeBS77BKpOmsnyfHhTV4Tk8XB9w/vKv8l6VZ/Pl8+fDSIyONf9kdvViEjUmX5HV+PXsFQgisNnW5A8bDAOsOZ+KZ745iV8oVfH0gHT/8dkmeo+H4pWJ8ZRozk55fjld/PIGs4rqBhfWdyiqRA9CB8wWoqTXgxOVi3PtJvEUvj3nyvPqXdb9aTa0B//ffY/hXvessKY35y7ip9yGEwMe7z6HPK1ssLmp5o8Sfz0dFjfEQYv15OD7ec14OnuevlOGH3y4hKbMIvzXyPoQQ+OvGZPzxs4P4x5bTiK0XcMuqa/HhzrNyIH1oRCgGhhgP7bg62mPbojGYZwoXXx9Ix3bTtv5uxllMT2eXIq+sRp7jAwDuGRyMv9/WDy9Ni5CX3dY/AGPrXadrWj8/jOzhbRFStI72uCO6S4Np2M009ipEBbnDUW2HLx4agrfuiMQfIv0xpW/dWX1RQe74cE50g4HxGns7jDKNRXtoZCiW39sfn94/CK6Oag5QtxHX/aB3fHw8JkyYYLFs8uTJWLhwYZPbVFdXo7q67he7pMTYLa/T6eTBTO3B/Fzt+ZwdkS3VQa8HtBoVFozthoLyGtzR399ivzLfhnWuG3Q4pqc3+vhrERXkBo1KoH+QGxLSi+THzX+lejirMTPKD5/uTcOB83XH+e1UEi4WVmLXqWwculAgz0xZpTPghf8ek9fbkpyNqlrjX9QDgtyQmFmMV/93AioIvPnLaVTqDCitqkG/AC12n82Dr9YRC8eHwcvFAe9utbzg2aHzV/DHz429JX/ZcBy/PDUCAHCxoG4wcVlFFTT1Tps0v/e3tqZg7WHj/BP3DgqAx+84rt+Ubw5l4rf0Irx2ay+LMNgS2SVVKCivkQ/TVdbocdo0oPNMTkmj++2/dqbig13GcURrD2VgzuAuEEIgt7QaPq6aZr/gjmYUoEZv+ftwMqsEWkc1Aj2cYDAI/HgsCz19O8ltAoy9NrV6AzRqO+w7e8XiOYd388TRS8XIK6tGUkY++nVxw4/1QuzRzEK4O9rhs/0XYK+S8JepEUi9UiaHVAA4nJaPCRHeEELgz2sSsSvF2DsX4umMhbd0Q355DV7aeAJ/6OcHHxd7vDipBzYmXpJnag70cMK9gwLxTqwxfE7s5YNgTyd8vj8do3t4w14yQKczYFofH7y1JQV6gwHzhnaBr5sLXBzsYKeS8PLUcOh0OoR41PU0TujlA7UkcFuUH974ueGEbQOC3GAH43OHeTshzNsJtbW1cFFLeGFyDxzNLMYbM/vI61ztjVt74f5hQRgaWncW4I34rLKlz8XmtLUOba2fJIRodX+oJEnYsGEDZs2a1eQ6PXv2xIMPPojFixfLyzZv3ozp06ejoqICTk5ODbZ59dVXsWTJkgbL16xZA2fnjnXdAeqYKmuB/zts/FKdHqTHpMC6X5PNmSpsvaiCWiWgM9Q7ldrLgD/2MOD/DtmhxrQ8TGvcLrVEgpOdQKVeQoSbAXoBnC0x/tXo6ySQWwkI0xTcjnYCrw3U48d0FfbmWP5lqVYJ6A2AwbTuGD8DJgYa8LcEOxggIcBZ4HKFBC+NQH51XdveG2Ycg7D4sB2q9Mblfx1QC29Hy/d9oRR4P9lObssj4Xr087T8iCjXAV+nqtDbQ2CUX9MfH5szVNh6ydj+O0P1za5rdqJQggpALw/jursuS/g5Q4VaATzTV48QV+BcCfCvE/ZyPd4aokdxDZBcKCHGR6BaD7zym538f6NWCfxjiB6HciWsPW+Hu0L1GOknUGsAMsoATw2wM0sFvQHwchTYlG6H3u4GPNbL+IWZXQH845gd3NTAXwfosfa8CoeuqOCqFnglWg+1CkgvBT4/YwcJwDP99Fhx0g45lXX1v72rHqklEo4VqDAtSI/JgQLLkuyQZVqni7NAdiWgF8b7oa4CAc4C++r9/4e6CjzdR48jeRK+SrWDnSRwa4gBg7wFOjXR2fj1WRUO5xmfY3yAARHuAitOGoPpS/1r4esEFNcALvZA/U6MlGIJQgAR7sb/h7wqQK0C3Orl1a/OqpBaIuGZvnq4mzLL/9JV2H5ZhWE+BnlMy5RAPaYGKXfsEF1fFRUVmDNnDoqLi6HVtnwCTUWePrB48WIsWrRIvl9SUoKgoCBMmjSpVW+yKTqdDrGxsZg4cSLUats9ZZR1aLwGK8/tRXpBBWZPHIqh9Sa56pFbht0fHcCd0V3wzaFMeWzKew+MQYC7Ez5J24/Tpgminr81GhcLK/H6z6dRaQoHD42PxJGMIpw1XTvk9TuisXxnKk5cNvYOLJoUgduGh2CWEFi+8xxWxNWdWVQ/FAFAYpEDBvXtCgNSERmoxd0DA/HyppMWAQUAuvYfCXs7CVUH4uVlVZ17YVdOGZ6ZEIYAdyfodDo89dkOOaAAgMGrG4aM6opHv05E/yB3/G16BD7YeQ4ni87hXJkKz909Ch7ODvhv4mUEezphSFfjJeBPZZVia3zda6XUeGDptGE4mFaA1Nwy3DMoEPZ2KvyQeAmlVbWYNywYaXkVWPjBPqgkCbufHYWSqlo8/cF++TmK3bpj2pRwfLr3AnDijFyPqOFj8cJ/k5GQXoQuod1hr5KgM6Qiws8VmYUVKK/Wo8fAUVjz0ykARbgodca0aYPw2k+n8NWJzEb3h5NFKjh3G4CxEb5Y+ksKDCIdhTXAjvJAHLpiHAxaqpNQ1rkvKnUGfHg4FTWm6dl/yu+MnMpCSBLwl6nhiD9fgJdu74tfknNw7MeTyJa80GNQb2TF1723SxXGmvcN0CK9oAJppbVIKzUuWzg+DO/vSEVaqYTPMz1xzDQ52qOju2HRhB6Ntt/M4VQuDq9JAgAsuDUGvf21yP/pFLp3dsEDzVyMbuLv+EyYhobzAk0VAvnlNSgs12Hah8b398dJlr8/HQU/F43aWgfzkZDWuu4hxc/PDzk5ORbLcnJyoNVqG+1FAQCNRgONpuHARbVafV12luv1vB0N62BZg/fv7Y9TWaUY0cPH4oO4dxcPHHtlMuxVEoK9XLAy7hw+mB2NkM7GAN3d11UOKRN7+0MA+Ol4tnz8f1pUF3hrHfFdwkVoHe0xsY8/sktr8LdNJxDk6YSHRnaTL6X+/JReGNbdeGr16n0XcNY0X8RjY7rh52NZuFhYiXe3Gy//ftfAIIwJ9wFgnAp8RlQAiipq8OvZPOxPK0TBVWf0vL3N2O2/91w+Dv/FeEj2kulkpsFdPXD4QiEOpxchLf8kjl8qwfFLJXjylh5Yc9gYrqprDfg24TICPZzw0sYTAIyDG/8zbxA2HTN+kQ/p6onfMgpx/FIJ9qcV4dEvf0N1rQEHLxQhKsgdy34xHqbqHeCOX5KzIQSgFwKxpy3PRgGAXSl5+OuMvvKXtNnm5Fz58NuawxfliDV/VDd8f+Qi4s/nY//5Qvxmqn9SZjEKq/RYd6TucEuwpzNySqpQXWuQL375zvZzGBXhj41H664F9XOy8X31D3JHUmYRXv2p7jDbiDAvHEorwCHT2TK9/bV4ZHQYHhltfPyW3n74648nkZRZhO8SjK8dGegmDy4FgGcnhcPV0R53flQX8B4Z3R2r9qejuFKHY5dK4GCvwq1RAXh6QjjU15jldFwvP0QGusHd2QEDQrwgSRKW3RHV7Db1teYzwd/BAV6uBgR6OEEIYFCo9zXbqWT8XDRqbR3aWrvrHlJiYmKwefNmi2WxsbGIiYm53i9N1CYDgj3keSauZh7g9+jo7pg/qptFiHlxcgQqqmuxYFyYPFHdf+YNwovfH8OIMG900thjXLgP/j03GgOC3SFJEuYMCYbGXoVx4T5yQDEzDxxMyS6VQ8rEXr7QOqrx9tYUAICzgx1mRAXA3dkBK+caByDeEuGDT39Nw69n8+T1GpNXVoPNx7MxqZc3LpYb2/vgiFAcvlDY4LLzi9YlIa+sGvYqCbUGga8OpFvMUPrr2Tws+d8Jea6ZR0d3wzrTxeIe+ypBvhjcL8nZ+CW5bj6aJf87icx6k+/97+hleJnmr1gwrjs+2XMe5/PKcTSzCAdMA4H9tI7ILqnCO/VO5TYPUg30cMIfovxxJrcU8efz8eGuVHlAcqVOj79uTEZ1rQF9ArRY/eAQeLo44Eh6IbaeyMbd0QGYtWIvUnLK8NhXRxqEO0e1Cp/cNxCT3t+DogodPF0c8OKUcNw9KAhfxqdjZdw5dHK0bzD4uou7EyL8XHE6uxRfmM7yeXBEV7y5+TSulFZD62gvT0749PgeWL7jLGZEBaCTxh6+Wo08idmKOdENTqtviqParsEVyW8EB3uV8aq8Au0yXTzZrhaHlLKyMqSmpsr309LSkJSUBE9PTwQHB2Px4sW4dOkSvvzySwDA448/jg8//BAvvPACHnroIezcuRPr1q3Dzz//3H7vgsiKrh6EGezljFUPDrFY5t1Jg89MM2uat6l/DRN7O5U8BX9ThnXzwpfx6fBwVmNAsAd6B2hxpbQabk5q/CHSX564amq95x3Zo+4MJx9XDR4dbRxgufKqy87/Y8tp9AsYhFKdBJUEjAv3QYiXM9LzjcHBwU6FGr1BnljrqfE9sC4hExcLK+UzbZ6d2BPvxp7Bt4eMh1C8XBwwJrwzfLWOiD+Xj9Jq47iY/5sagdiTOcgsqMDIMG9sSLokX2wy2NMZmYUVcq8HAEzq7YejmcXYm5qH+V8moLBCh1BvF/wh0h8f7Kz7LBob3hlxKVfgqrHHZ/cPhsbeDv0D3QEApVV184IAwNYTxt7dB0eEorPptNghoZ4YEuoJnU6HyYEGbEy3w27Te7t7UCDWmQ7PTertBx+tI9Y9FoO0vHKM6dlZ/iK+f3hX3D+8a5P/h38aF4anvk001tRehQm9fLEp6TLiUq5gYm8/OfwunNADw7p5oU8XY+/czP5d8PbWFIR4OWNCr+szIVx746zL1B5aHFISEhIwblzdTKHmsSP3338/Vq9ejaysLGRk1M1dEBoaip9//hnPPPMMli9fjsDAQHz66aeYPHlyOzSfyHZM7O2L+aNCMSTUC3YqCc4O9nj11j7NbtPLX4tVDw6GEAKjenSG2k6Fr+udMWKehCujoAJv/mLsbQn1doGTgx3euiMSO1NyMaWPH9R2Kvzhg70AjGFn/qhucHW0xxLTVWY7u2qwYFwYnBzs8M/YM6io0eOB4V2htlOhX6AbNiwYgdd+OonoYHc8PqY7Hh9TN7GXRq3Ct4cyERXohrfujMLrP520mHysT4AWc4YGY29qHnJNPSWvzeyDipq6uUXmjwrFoonh+HdcKib29kW4n3GejYFdPeSABQATevli+yljQInwc8WMqMbn3xnlJ3CkxAmZhZW4bUAXLLs9EofSCnAhvwK3Rxvn5ujp62pxZeDf4w/9/LEy7hxOZZVgbM/OcHVU45GR3VBeXWsx2ZkkSYipN5HZQyNC0Uljjz9E+vPUW7IpLQ4pY8eORXMnBDU2m+zYsWORmJjY0pcionrUdir8ZXrvFm83LtzyL+8A97pTegYEe2BIV0/8ffMpbDH1LvT2N37xDu3mhaGmK04LIRDq7YK0vHL8485IODnY4e5BQXgv9gxKqmpxS7gPVCoJj4zqhodHhqKsutZifpswn0748iHL3iWz12f2xcMjQ9G9cydIkoSlt/fD1OW/oqy6FhN7+8LeToVp/fyx5pGheG/7GQzu6olRPTqjptaAx0Z3w4Bgd3myP/O1Ysx8XB3x459H4OTlEng4O6CzqwbbT+XAwV6FT+4bJF+q4Gr2KuCzedFIulSK2wZ0gUol4aP7BiI1t8xi3pCWUqkkvH1nJN7dloJnJvYEYOztqt/j1RgnB7tme2iIblaKPLuHiK4fP23dgPVe/lr8cVgIvjqQjgzTHCq9G7mekCRJ+PqRocgtqZLH6bho7PHc5HD8M/YM5tSbqVSSpBZNwGdvp7K4cFuQpzN2PjcGXx/IwKz+AfLy4WHeFhP0OdirsHhar2s+f4SfFhF+de/pq4eHINjTGcFezU9nEOrtgp7+7k0+T2v17eLW4HAgETWOIYXIxphnHQWAXn6ucHKww6YFI/Dm5pPYfeIiJjYx5qGLuxO6uFuekTcvpivmxXRt9zb6uDpikamnob2NauRq2kSkTAwpRDbG3VkNNyc1iit18pk5Hi4OeHNWH2x2SEfINXoYiIhuFIYUIhsjSRI+vm8g8stqEOTJQEJEysWQQmSDhnXzuvZKRERW1uKrIBMRERHdCAwpREREpEgMKURERKRIDClERESkSAwpREREpEgMKURERKRIDClERESkSAwpREREpEgMKURERKRIDClERESkSAwpREREpEgMKURERKRIDClERESkSB3iKshCCABASUlJuz6vTqdDRUUFSkpKoFar2/W5OxLWgTUAWAMz1oE1AFgDs7bWwfy9bf4eb6kOEVJKS0sBAEFBQVZuCREREbVUaWkp3NzcWrydJFobb24gg8GAy5cvw9XVFZIktdvzlpSUICgoCJmZmdBqte32vB0N68AaAKyBGevAGgCsgVlb6yCEQGlpKQICAqBStXyESYfoSVGpVAgMDLxuz6/Vam16JzRjHVgDgDUwYx1YA4A1MGtLHVrTg2LGgbNERESkSAwpREREpEg2HVI0Gg1eeeUVaDQaazfFqlgH1gBgDcxYB9YAYA3MrF2HDjFwloiIiGyPTfekEBERkXIxpBAREZEiMaQQERGRIjGkEBERkSLZdEhZsWIFunbtCkdHRwwdOhSHDh2ydpNaZenSpRg8eDBcXV3h4+ODWbNmISUlxWKdsWPHQpIki3+PP/64xToZGRmYPn06nJ2d4ePjg+effx61tbUW68TFxSE6OhoajQZhYWFYvXr19X57v9urr77a4D1GRETIj1dVVWHBggXw8vJCp06dcMcddyAnJ8fiOTp6Dbp27dqgBpIkYcGCBQBuzv1gz549mDFjBgICAiBJEjZu3GjxuBACf/vb3+Dv7w8nJydMmDABZ8+etVinoKAAc+fOhVarhbu7Ox5++GGUlZVZrHPs2DGMGjUKjo6OCAoKwltvvdWgLevXr0dERAQcHR3Rr18/bN68ud3fb1Oaq4NOp8OLL76Ifv36wcXFBQEBAZg3bx4uX75s8RyN7T/Lli2zWEfJdbjWvvDAAw80eH9TpkyxWKej7wvXqkFjnw+SJOHtt9+W11HUfiBs1Nq1a4WDg4P4/PPPxYkTJ8T8+fOFu7u7yMnJsXbTWmzy5Mli1apVIjk5WSQlJYlp06aJ4OBgUVZWJq8zZswYMX/+fJGVlSX/Ky4ulh+vra0Vffv2FRMmTBCJiYli8+bNwtvbWyxevFhe5/z588LZ2VksWrRInDx5UnzwwQfCzs5ObNmy5Ya+36a88sorok+fPhbv8cqVK/Ljjz/+uAgKChI7duwQCQkJYtiwYWL48OHy4zdDDXJzcy3ef2xsrAAgdu3aJYS4OfeDzZs3i7/85S/ihx9+EADEhg0bLB5ftmyZcHNzExs3bhRHjx4Vt956qwgNDRWVlZXyOlOmTBFRUVHiwIED4tdffxVhYWFi9uzZ8uPFxcXC19dXzJ07VyQnJ4tvv/1WODk5iY8//lheZ9++fcLOzk689dZb4uTJk+Lll18WarVaHD9+/LrXQIjm61BUVCQmTJggvvvuO3H69GkRHx8vhgwZIgYOHGjxHCEhIeK1116z2D/qf44ovQ7X2hfuv/9+MWXKFIv3V1BQYLFOR98XrlWD+u89KytLfP7550KSJHHu3Dl5HSXtBzYbUoYMGSIWLFgg39fr9SIgIEAsXbrUiq1qH7m5uQKA2L17t7xszJgx4umnn25ym82bNwuVSiWys7PlZStXrhRarVZUV1cLIYR44YUXRJ8+fSy2u+eee8TkyZPb9w200iuvvCKioqIafayoqEio1Wqxfv16edmpU6cEABEfHy+EuDlqcLWnn35adO/eXRgMBiHEzb8fXP2hbDAYhJ+fn3j77bflZUVFRUKj0Yhvv/1WCCHEyZMnBQBx+PBheZ1ffvlFSJIkLl26JIQQ4t///rfw8PCQayCEEC+++KIIDw+X7999991i+vTpFu0ZOnSoeOyxx9r1Pf4ejX05Xe3QoUMCgEhPT5eXhYSEiPfee6/JbTpSHZoKKTNnzmxym5ttX/g9+8HMmTPFLbfcYrFMSfuBTR7uqampwZEjRzBhwgR5mUqlwoQJExAfH2/FlrWP4uJiAICnp6fF8m+++Qbe3t7o27cvFi9ejIqKCvmx+Ph49OvXD76+vvKyyZMno6SkBCdOnJDXqV8z8zpKqtnZs2cREBCAbt26Ye7cucjIyAAAHDlyBDqdzqL9ERERCA4Oltt/s9TArKamBl9//TUeeughiwtz2sJ+YJaWlobs7GyL9rq5uWHo0KEW/+/u7u4YNGiQvM6ECROgUqlw8OBBeZ3Ro0fDwcFBXmfy5MlISUlBYWGhvE5HqQtg/JyQJAnu7u4Wy5ctWwYvLy8MGDAAb7/9tsWhvpuhDnFxcfDx8UF4eDieeOIJ5Ofny4/Z2r6Qk5ODn3/+GQ8//HCDx5SyH3SICwy2t7y8POj1eosPYgDw9fXF6dOnrdSq9mEwGLBw4UKMGDECffv2lZfPmTMHISEhCAgIwLFjx/Diiy8iJSUFP/zwAwAgOzu70XqYH2tunZKSElRWVsLJyel6vrVrGjp0KFavXo3w8HBkZWVhyZIlGDVqFJKTk5GdnQ0HB4cGH8i+vr7XfH/mx5pbRyk1qG/jxo0oKirCAw88IC+zhf2gPnObG2tv/ffj4+Nj8bi9vT08PT0t1gkNDW3wHObHPDw8mqyL+TmUpKqqCi+++CJmz55tcdG4p556CtHR0fD09MT+/fuxePFiZGVl4Z///CeAjl+HKVOm4Pbbb0doaCjOnTuHl156CVOnTkV8fDzs7Oxsbl/44osv4Orqittvv91iuZL2A5sMKTezBQsWIDk5GXv37rVY/uijj8o/9+vXD/7+/hg/fjzOnTuH7t273+hmXhdTp06Vf46MjMTQoUMREhKCdevWKeqL80b57LPPMHXqVAQEBMjLbGE/oObpdDrcfffdEEJg5cqVFo8tWrRI/jkyMhIODg547LHHsHTp0ptievh7771X/rlfv36IjIxE9+7dERcXh/Hjx1uxZdbx+eefY+7cuXB0dLRYrqT9wCYP93h7e8POzq7BmR05OTnw8/OzUqva7sknn8RPP/2EXbt2ITAwsNl1hw4dCgBITU0FAPj5+TVaD/Njza2j1WoVGQLc3d3Rs2dPpKamws/PDzU1NSgqKrJYp/7/+c1Ug/T0dGzfvh2PPPJIs+vd7PuBuc3N/a77+fkhNzfX4vHa2loUFBS0y76hpM8Uc0BJT09HbGysRS9KY4YOHYra2lpcuHABwM1TB7Nu3brB29vbYv+3lX3h119/RUpKyjU/IwDr7gc2GVIcHBwwcOBA7NixQ15mMBiwY8cOxMTEWLFlrSOEwJNPPokNGzZg586dDbrhGpOUlAQA8Pf3BwDExMTg+PHjFr+g5g+x3r17y+vUr5l5HaXWrKysDOfOnYO/vz8GDhwItVpt0f6UlBRkZGTI7b+ZarBq1Sr4+Phg+vTpza53s+8HoaGh8PPzs2hvSUkJDh48aPH/XlRUhCNHjsjr7Ny5EwaDQQ5xMTEx2LNnD3Q6nbxObGwswsPD4eHhIa+j5LqYA8rZs2exfft2eHl5XXObpKQkqFQq+RDIzVCH+i5evIj8/HyL/d8W9gXA2NM6cOBAREVFXXNdq+4HLRpmexNZu3at0Gg0YvXq1eLkyZPi0UcfFe7u7hZnNXQUTzzxhHBzcxNxcXEWp4xVVFQIIYRITU0Vr732mkhISBBpaWli06ZNolu3bmL06NHyc5hPPZ00aZJISkoSW7ZsEZ07d2701NPnn39enDp1SqxYsUJRp98+++yzIi4uTqSlpYl9+/aJCRMmCG9vb5GbmyuEMJ6CHBwcLHbu3CkSEhJETEyMiImJkbe/GWoghPFMteDgYPHiiy9aLL9Z94PS0lKRmJgoEhMTBQDxz3/+UyQmJspnrSxbtky4u7uLTZs2iWPHjomZM2c2egrygAEDxMGDB8XevXtFjx49LE47LSoqEr6+vuK+++4TycnJYu3atcLZ2bnBKZf29vbinXfeEadOnRKvvPLKDT0Fubk61NTUiFtvvVUEBgaKpKQki88J8xka+/fvF++9955ISkoS586dE19//bXo3LmzmDdvXoepQ3M1KC0tFc8995yIj48XaWlpYvv27SI6Olr06NFDVFVVyc/R0feFa/0+CGE8hdjZ2VmsXLmywfZK2w9sNqQIIcQHH3wggoODhYODgxgyZIg4cOCAtZvUKgAa/bdq1SohhBAZGRli9OjRwtPTU2g0GhEWFiaef/55i/kxhBDiwoULYurUqcLJyUl4e3uLZ599Vuh0Oot1du3aJfr37y8cHBxEt27d5NdQgnvuuUf4+/sLBwcH0aVLF3HPPfeI1NRU+fHKykrxpz/9SXh4eAhnZ2dx2223iaysLIvn6Og1EEKIrVu3CgAiJSXFYvnNuh/s2rWr0f3//vvvF0IYT0P+61//Knx9fYVGoxHjx49vUJv8/Hwxe/Zs0alTJ6HVasWDDz4oSktLLdY5evSoGDlypNBoNKJLly5i2bJlDdqybt060bNnT+Hg4CD69Okjfv755+v2vq/WXB3S0tKa/Jwwz6Fz5MgRMXToUOHm5iYcHR1Fr169xJtvvmnxBS6EsuvQXA0qKirEpEmTROfOnYVarRYhISFi/vz5Df4w7ej7wrV+H4QQ4uOPPxZOTk6iqKiowfZK2w8kIYRoWd8LERER0fVnk2NSiIiISPkYUoiIiEiRGFKIiIhIkRhSiIiISJEYUoiIiEiRGFKIiIhIkRhSiIiISJEYUoiIiEiRGFKIiIhIkRhSiIiISJEYUoiIiEiRGFKIiIhIkf4fdkEHr7vRcVAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:03:42.243597600Z",
     "start_time": "2024-07-29T08:03:42.228375900Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:36:32.325554Z",
     "iopub.status.busy": "2025-02-04T09:36:32.325062Z",
     "iopub.status.idle": "2025-02-04T09:36:32.339493Z",
     "shell.execute_reply": "2025-02-04T09:36:32.338610Z",
     "shell.execute_reply.started": "2025-02-04T09:36:32.325518Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4502, 0.5498])\n"
     ]
    }
   ],
   "source": [
    "#下面的例子是为了说明temperature\n",
    "my_tensor = torch.tensor([0.4,0.6]) #这里是logits\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:17:00.048904100Z",
     "start_time": "2024-07-29T08:17:00.043755800Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:36:33.676449Z",
     "iopub.status.busy": "2025-02-04T09:36:33.675938Z",
     "iopub.status.idle": "2025-02-04T09:36:33.682011Z",
     "shell.execute_reply": "2025-02-04T09:36:33.681317Z",
     "shell.execute_reply.started": "2025-02-04T09:36:33.676415Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4750, 0.5250])\n"
     ]
    }
   ],
   "source": [
    "my_tensor = torch.tensor([0.2,0.3])  #现在 temperature是2\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:06:39.649844900Z",
     "start_time": "2024-07-29T08:06:39.642147800Z"
    },
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-02-04T09:36:34.187716Z",
     "iopub.status.busy": "2025-02-04T09:36:34.187229Z",
     "iopub.status.idle": "2025-02-04T09:36:34.202077Z",
     "shell.execute_reply": "2025-02-04T09:36:34.201362Z",
     "shell.execute_reply.started": "2025-02-04T09:36:34.187685Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "概率分布: tensor([0.1000, 0.4500, 0.3500, 0.1000])\n",
      "抽取的样本索引: tensor([2])\n",
      "每个样本对应的概率: tensor([0.3500])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# 创建一个概率分布，表示每个类别被选中的概率\n",
    "# 这里我们有一个简单的四个类别的概率分布\n",
    "prob_dist = torch.tensor([0.1, 0.45, 0.35, 0.1])\n",
    "\n",
    "# 使用 multinomial 进行抽样\n",
    "# num_samples 表示要抽取的样本数量\n",
    "num_samples = 5\n",
    "\n",
    "# 抽取样本，随机抽样，概率越高，抽到的概率就越高\n",
    "samples = torch.multinomial(prob_dist, 1, replacement=True)\n",
    "\n",
    "print(\"概率分布:\", prob_dist)\n",
    "print(\"抽取的样本索引:\", samples)\n",
    "\n",
    "# 显示每个样本对应的概率\n",
    "print(\"每个样本对应的概率:\", prob_dist[samples])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-29T08:51:40.360874200Z",
     "start_time": "2024-07-29T08:51:38.070276100Z"
    },
    "execution": {
     "iopub.execute_input": "2025-02-04T09:36:36.219247Z",
     "iopub.status.busy": "2025-02-04T09:36:36.218755Z",
     "iopub.status.idle": "2025-02-04T09:36:37.763348Z",
     "shell.execute_reply": "2025-02-04T09:36:37.762598Z",
     "shell.execute_reply.started": "2025-02-04T09:36:36.219213Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1243/612744915.py:26: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
      "  0%|          | 0/1000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All: I"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 1/1000 [00:00<03:53,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " cannot now a strange report of any of your sorrow's eye,\n",
      "And be it f"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 70/1000 [00:00<00:03, 263.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "orth to come of this.\n",
      "\n",
      "Clown:\n",
      "Alack, the vile the common of the sea-should success sh"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 155/1000 [00:00<00:01, 467.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all put\n",
      "He that runs for his propert to repent;\n",
      "Go, crave I need of men,\n",
      "Should wish"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 239/1000 [00:00<00:01, 591.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " me not with me; let him not confederation of my company.\n",
      "\n",
      "KING RICHARD III"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███▏      | 314/1000 [00:00<00:01, 640.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ":\n",
      "Why, she, my lord?\n",
      "\n",
      "KING RICHARD II:\n",
      "What says he?\n",
      "\n",
      "Nurse:\n",
      "What is the"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▊      | 386/1000 [00:00<00:00, 649.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " frowns of his majesty\n",
      "With all my house to assurance of his love\n",
      "Than thou must be"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 469/1000 [00:00<00:00, 703.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " here.\n",
      "\n",
      "Second Citizen:\n",
      "And if thou weep'st off the heart,\n",
      "Or else it will"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 543/1000 [00:00<00:00, 694.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " not be long.\n",
      "The gods be seen.\n",
      "\n",
      "First Senator:\n",
      "How canst not say you he"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 615/1000 [00:01<00:00, 684.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ar.\n",
      "\n",
      "CLARENCE:\n",
      "And therefore has he dispatch of him,\n",
      "And leave you thus I told you "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|██████▉   | 698/1000 [00:01<00:00, 726.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stay.\n",
      "\n",
      "EDWARD:\n",
      "Now the report it. My dear king,\n",
      "For fearing you would seem still "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 779/1000 [00:01<00:00, 750.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "will not be proud to have a stubborn and fortune enough not what the heart of heaven,"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▋ | 864/1000 [00:01<00:00, 777.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "To choose but that the foolish women, and leave me so much some prick'd in a house o"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 949/1000 [00:01<00:00, 797.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f Lancaster.\n",
      "\n",
      "PROSPERO:\n",
      "Thou hast lost too late!\n",
      "\n",
      "M"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:01<00:00, 655.56it/s]\n"
     ]
    }
   ],
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1) #bacth_size=1, seq_len长度是多少都可以 （1,5）\n",
    "    hidden = None\n",
    "    text_generated = [] #用来保存生成的文本\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len)) # 进度条\n",
    "    print(start_string, end=\"\")\n",
    "    # no_grad是一个上下文管理器，用于指定在其中的代码块中不需要计算梯度。在这个区域内，不会记录梯度信息，用于在生成文本时不影响模型权重。\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:#控制进度条\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样，较高的温度会增加预测结果的多样性，较低的温度则更加保守。\n",
    "            #取-1的目的是只要最后，拼到原有的输入上\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1) #算为概率分布\n",
    "            idx = torch.multinomial(probs, 1).item() #从概率分布中抽取一个样本,取概率较大的那些\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1) #把idx转为tensor\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \" #这里就是开头，什么都可以\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
